var request = require('request');
var Iconv = require('iconv-lite');
var fs = require("fs");



class Spider {

    constructor(url, headers = {},sleepTime = 100) {
        this.url = url;
        this.headers = headers;
        this.sleepTime = sleepTime;

        this.pages = 22;
        this.curPage = 1;

        this.urls = [];
        this.fails = [];

        this.fun_detail = null;
    }

    // 获取全部爬取url链接 fun_urls return array
    start(fun_urls, fun_pages, fun_detail) {

        this.fun_urls = fun_urls;
        this.fun_pages = fun_pages;
        this.fun_detail = fun_detail;

        this.pageNext();
    }

    pageNext(page) {
        // 如果page不存在，默认this.curPage
        if (!page) {
            page = this.curPage;
        }

        this.curPage = page;
        let targe_url = this.url.replace("*PageReplace*", page);

        // page
        if (page > this.pages) {
            console.log("page all spider");
            setTimeout(() => {
                this.failsHandel();
            }, 50000);
            return;
        } else {
            console.log("  " + page + "/" + this.pages);
        }

        // headers set
        var headers = {};
        if (this.headers['Referer']) {
            this.headers['Referer'] = targe_url;
        }

        // 页面爬取
        this.spider(this, targe_url, this.headers).then((body) => {
            // console.log(body);
            console.log("---success--" + targe_url);
            var success_urls = this.fun_urls(body);

            this.urls = this.urls.concat(success_urls);

            if (page == 1) {
                this.pages = this.fun_pages(body);
            }



            this.getDetail(success_urls, this.fun_detail, headers);


            page += 1;
            this.pageNext(page);


        }, (err) => {
            console.log(err);
            console.log(targe_url);
            this.sleep(1000);
            this.pageNext(page);

        });
    }

    // promise return
    spider(_self, url, headers = {}) {
        var q = new Promise((resolve, reject) => {
            request({ "url": url, "headers": headers }, function (error, response, body) {
                if (!error && response.statusCode == 200) {
                    resolve(_self.decodeU(response, body));
                } else {
                    reject(error);
                }
            });
        });

        return q;

    }



    // 获取详情页数据
    getDetail(urls, fun_detail) {
        let length = urls.length;

        for (let index in urls) {
            let item = urls[index];

            if(this.sleepTime){
                this.sleep(this.sleepTime);
            }
            this.spider(this, item).then((body) => {
                console.log("detail url success:" + item);
                fun_detail(item, body);

            }, (err) => {
                console.log("detail url error:" + item);
                this.fails.push(item);
            });
            // this.sleep(1000);
        }
    }

    // 错误处理
    failsHandel() {
        if (this.fails.length == 0) {
            console.log("-------------all success-----------------");
        } else {
            console.log("---------has " + this.fails.length + " url get fails----------");
            this.logger(this.fails);
            this.getDetail(this.fails, this.fun_detail);
        }
    }

    logger(data) {
        if (typeof (data) == "object") {
            data = JSON.stringify(data);
        } else {
            data = data.toString();
        }
        fs.writeFile('logger.txt', data, function (err) {
            if (err) {
                cosnole.log(err);
            }
        });
    }

    sleep(time) {
        var waitUntil = new Date(new Date().getTime() + time);
        while (waitUntil > new Date()) {
            // console.log('set time out '+time);
        }
    }
    // 汉字乱码问题
    decodeU(response, body) {
        var contentType = response.headers['content-type'];
        if (contentType.indexOf("Charset") != -1) {
            var unicode = contentType.split("Charset=")[1];
            if (unicode != "utf-8") {
                body = Iconv.decode(body, unicode).toString();
            }
        }

        return body;
    }

}

module.exports = Spider;